{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 简单的反向传播的实现，计算图……\n",
    "\n",
    "## 需求分析\n",
    "\n",
    "- 根据链式法则需要将所有的局部梯度相乘\n",
    "- 局部梯度的计算需要知道区分运算，还需要知道运算数\n",
    "- 必须自己定义一个类用运算符重载和属性实现这些功能（Value）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "class Value():\n",
    "    def __init__(self, data, child=()) -> None:\n",
    "        '''\n",
    "        data:当前变量的数值\n",
    "        child:运算数（用于链式法则梯度计算）\n",
    "        不同运算的区分可以在不同的函数里设置不同的_backward实现\n",
    "        '''\n",
    "        self.data = data\n",
    "        self._child = child\n",
    "        self.grad = 0.0\n",
    "        self._backward = None\n",
    "\n",
    "    def __repr__(self) -> str:\n",
    "        return f\"Value({self.data})\"\n",
    "\n",
    "    def fornotvalue(fun):\n",
    "        def wrapper(*args):\n",
    "            args = [arg if isinstance(arg, Value) else Value(arg) for arg in args]\n",
    "            return fun(*args)\n",
    "        return wrapper\n",
    "\n",
    "    @fornotvalue\n",
    "    def __add__(self, other):\n",
    "        out = Value(self.data + other.data, (self, other))\n",
    "        \n",
    "        def _backward():\n",
    "            # 这里的变量都是指针，因为对于此函数都是外部的\n",
    "            self.grad += 1.0 * out.grad # 链式法则(L)\n",
    "            other.grad += 1.0 * out.grad # 链式法则(R)\n",
    "        \n",
    "        out._backward = _backward # (O)\n",
    "        return out\n",
    "    def __radd__(self, other):\n",
    "        return self + other\n",
    "    \n",
    "    def __neg__(self):\n",
    "        return self * -1\n",
    "\n",
    "    def __sub__(self, other):\n",
    "        return self + (-other)\n",
    "\n",
    "    def __rsub__(self, other):\n",
    "        return -self + other\n",
    "\n",
    "    @fornotvalue\n",
    "    def __mul__(self, other):\n",
    "        out = Value(self.data * other.data, (self, other))\n",
    "        \n",
    "        def _backward():\n",
    "            # 这里的变量都是指针，因为对于此函数都是外部的\n",
    "            self.grad += other.data * out.grad # 链式法则(L)\n",
    "            other.grad += self.data * out.grad # 链式法则(R)\n",
    "        \n",
    "        out._backward = _backward # (O)\n",
    "        return out\n",
    "    def __rmul__(self, other):\n",
    "        return self * other\n",
    "    @fornotvalue\n",
    "    def __pow__(self, other):\n",
    "        assert isinstance(other.data, (float, int))\n",
    "        y = self.data**other.data\n",
    "        out = Value(y, (self, other))\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += other.data * self.data**(other.data - 1) * out.grad\n",
    "            other.grad += y*math.log(other.data, math.e) * out.grad\n",
    "        \n",
    "        out._backward = _backward # (O)\n",
    "        return out\n",
    "\n",
    "    def __truediv__(self, other):\n",
    "        return self * other**-1\n",
    "\n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        x_2 = math.exp(x * 2)\n",
    "        t = (x_2 - 1) / (x_2 + 1)\n",
    "        out = Value(t, (self,))\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += (1.0 - t**2) * out.grad\n",
    "\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "\n",
    "    def exp(self):\n",
    "        x = self.data\n",
    "        y = math.exp(x)\n",
    "        out = Value(y, (self,))\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += y * out.grad\n",
    "\n",
    "        out._backward = _backward\n",
    "        return out\n",
    "\n",
    "    def backward(self):\n",
    "        self.grad = 1.0\n",
    "        allval = []\n",
    "        def bulid(Value):\n",
    "            allval.append(Value)\n",
    "            for child in Value._child:\n",
    "                if (child not in allval) and (child._backward!=None):\n",
    "                    bulid(child)\n",
    "        bulid(self)\n",
    "        # print(len(allval))\n",
    "        for val in allval:\n",
    "            val._backward()           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from graphviz import Digraph\n",
    "def trace(root):\n",
    "    nodes, edges = set(), set()\n",
    "    def build(v):\n",
    "        if v not in nodes:\n",
    "            nodes.add(v)\n",
    "            for child in v._child:\n",
    "                edges.add((child, v))\n",
    "                build(child)\n",
    "    build(root)\n",
    "    return nodes, edges\n",
    "\n",
    "def draw_dot(root, format='svg', rankdir='LR'):\n",
    "    \"\"\"\n",
    "    format: png | svg | ...\n",
    "    rankdir: TB (top to bottom graph) | LR (left to right)\n",
    "    \"\"\"\n",
    "    assert rankdir in ['LR', 'TB']\n",
    "    nodes, edges = trace(root)\n",
    "    dot = Digraph(format=format, graph_attr={'rankdir': rankdir}) #, node_attr={'rankdir': 'TB'})\n",
    "    \n",
    "    for n in nodes:\n",
    "        dot.node(name=str(id(n)), label = \"{ data %.4f | grad %.4f }\" % (n.data, n.grad), shape='record')\n",
    "        # if n._op:\n",
    "        #     # dot.node(name=str(id(n)) + n._op, label=n._op)\n",
    "        #     dot.node(name=str(id(n)))\n",
    "        #     # dot.edge(str(id(n)) + n._op, str(id(n)))\n",
    "        #     dot.edge(str(id(n)), str(id(n)))\n",
    "    \n",
    "    for n1, n2 in edges:\n",
    "        # dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "        dot.edge(str(id(n1)), str(id(n2)))\n",
    "    \n",
    "    return dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x1 = Value(10)\n",
    "# x2 = Value(5)\n",
    "# x3 = Value(3)\n",
    "# b = Value(1)\n",
    "\n",
    "# def f(x1, x2, x3, b):\n",
    "#     return 35*x1**3 + 567*x2**2 + 6*x3 + b\n",
    "# def tanh(x):\n",
    "#     return x.tanh()\n",
    "# y = f(x1, x2, tanh(x3), b)\n",
    "# y.backward()\n",
    "# print(f\"dy/dx1:{x1.grad}\\ndy/dx2:{ x2.grad}\\ndy/dx3:{x3.grad}\\ndy/db:{b.grad}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n",
       " -->\n",
       "<!-- Pages: 1 -->\n",
       "<svg width=\"1204pt\" height=\"237pt\"\n",
       " viewBox=\"0.00 0.00 1203.50 237.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 233)\">\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-233 1199.5,-233 1199.5,4 -4,4\"/>\n",
       "<!-- 1979235280392 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1979235280392</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"216.5,-192.5 216.5,-228.5 383.5,-228.5 383.5,-192.5 216.5,-192.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"256.75\" y=\"-205.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"297,-192.75 297,-228.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"340.25\" y=\"-205.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;3.0000</text>\n",
       "</g>\n",
       "<!-- 1979236228936 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>1979236228936</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"421.75,-165.5 421.75,-201.5 593.25,-201.5 593.25,-165.5 421.75,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"464.25\" y=\"-178.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"506.75,-165.75 506.75,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"550\" y=\"-178.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;3.0000</text>\n",
       "</g>\n",
       "<!-- 1979235280392&#45;&gt;1979236228936 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1979235280392&#45;&gt;1979236228936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M383.99,-199.6C392.69,-198.45 401.58,-197.29 410.4,-196.13\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"410.56,-199.63 420.02,-194.86 409.65,-192.69 410.56,-199.63\"/>\n",
       "</g>\n",
       "<!-- 1979236221000 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>1979236221000</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"214.25,-137.5 214.25,-173.5 385.75,-173.5 385.75,-137.5 214.25,-137.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"256.75\" y=\"-150.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;5.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"299.25,-137.75 299.25,-173.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"342.5\" y=\"-150.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;3.0000</text>\n",
       "</g>\n",
       "<!-- 1979236221000&#45;&gt;1979236228936 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>1979236221000&#45;&gt;1979236228936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M386.07,-167.09C394.02,-168.17 402.12,-169.28 410.17,-170.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"409.62,-173.83 420,-171.71 410.56,-166.9 409.62,-173.83\"/>\n",
       "</g>\n",
       "<!-- 1979265013640 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>1979265013640</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"827.75,-28.5 827.75,-64.5 997,-64.5 997,-28.5 827.75,-28.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"868\" y=\"-41.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 0.5000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"908.25,-28.75 908.25,-64.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"952.62\" y=\"-41.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 10.0000</text>\n",
       "</g>\n",
       "<!-- 1979265015560 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>1979265015560</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"1033,-55.5 1033,-91.5 1195.5,-91.5 1195.5,-55.5 1033,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1073.25\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 5.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"1113.5,-55.75 1113.5,-91.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"1154.5\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 1979265013640&#45;&gt;1979265015560 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>1979265013640&#45;&gt;1979265015560</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M997.27,-57.83C1005.26,-58.91 1013.39,-60.01 1021.44,-61.1\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1020.89,-64.55 1031.26,-62.43 1021.82,-57.62 1020.89,-64.55\"/>\n",
       "</g>\n",
       "<!-- 1979265015432 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>1979265015432</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"827.75,-83.5 827.75,-119.5 997,-119.5 997,-83.5 827.75,-83.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"871.38\" y=\"-96.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 10.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"915,-83.75 915,-119.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"956\" y=\"-96.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 1979265015432&#45;&gt;1979265015560 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>1979265015432&#45;&gt;1979265015560</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M997.27,-89.75C1005.26,-88.63 1013.39,-87.49 1021.44,-86.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"1021.85,-89.84 1031.27,-84.99 1020.88,-82.91 1021.85,-89.84\"/>\n",
       "</g>\n",
       "<!-- 1979236222600 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>1979236222600</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"7.88,-165.5 7.88,-201.5 170.38,-201.5 170.38,-165.5 7.88,-165.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.12\" y=\"-178.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 5.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"88.38,-165.75 88.38,-201.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"129.38\" y=\"-178.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 3.0000</text>\n",
       "</g>\n",
       "<!-- 1979236222600&#45;&gt;1979236221000 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>1979236222600&#45;&gt;1979236221000</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M170.58,-172.71C181.16,-171.3 192.08,-169.83 202.89,-168.38\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.14,-171.88 212.58,-167.08 202.21,-164.94 203.14,-171.88\"/>\n",
       "</g>\n",
       "<!-- 1979265013896 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>1979265013896</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"216.5,-0.5 216.5,-36.5 383.5,-36.5 383.5,-0.5 216.5,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"256.75\" y=\"-13.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"297,-0.75 297,-36.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"340.25\" y=\"-13.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;1.0000</text>\n",
       "</g>\n",
       "<!-- 1979265013448 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>1979265013448</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"421.75,-55.5 421.75,-91.5 593.25,-91.5 593.25,-55.5 421.75,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"464.25\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"506.75,-55.75 506.75,-91.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"550\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;1.0000</text>\n",
       "</g>\n",
       "<!-- 1979265013896&#45;&gt;1979265013448 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1979265013896&#45;&gt;1979265013448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M370,-36.97C388.23,-41.85 408.05,-47.15 426.79,-52.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"425.65,-55.49 436.22,-54.69 427.46,-48.72 425.65,-55.49\"/>\n",
       "</g>\n",
       "<!-- 1979265012936 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>1979265012936</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"629.25,-55.5 629.25,-91.5 791.75,-91.5 791.75,-55.5 629.25,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"669.5\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"709.75,-55.75 709.75,-91.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"750.75\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 1979265012936&#45;&gt;1979265015432 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>1979265012936&#45;&gt;1979265015432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M792.22,-84.81C800.1,-85.91 808.15,-87.04 816.15,-88.16\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"815.53,-91.61 825.92,-89.53 816.5,-84.68 815.53,-91.61\"/>\n",
       "</g>\n",
       "<!-- 1979265013448&#45;&gt;1979265012936 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>1979265013448&#45;&gt;1979265012936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M593.74,-73.5C601.58,-73.5 609.54,-73.5 617.43,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"617.41,-77 627.41,-73.5 617.41,-70 617.41,-77\"/>\n",
       "</g>\n",
       "<!-- 1979236221384 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>1979236221384</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"7.88,-55.5 7.88,-91.5 170.38,-91.5 170.38,-55.5 7.88,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"48.12\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"88.38,-55.75 88.38,-91.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"129.38\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 1.0000</text>\n",
       "</g>\n",
       "<!-- 1979265012040 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>1979265012040</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"214.25,-55.5 214.25,-91.5 385.75,-91.5 385.75,-55.5 214.25,-55.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"256.75\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"299.25,-55.75 299.25,-91.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"342.5\" y=\"-68.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;1.0000</text>\n",
       "</g>\n",
       "<!-- 1979236221384&#45;&gt;1979265012040 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>1979236221384&#45;&gt;1979265012040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M170.58,-73.5C181.16,-73.5 192.08,-73.5 202.89,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"202.57,-77 212.57,-73.5 202.57,-70 202.57,-77\"/>\n",
       "</g>\n",
       "<!-- 1979265014472 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>1979265014472</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"426.25,-0.5 426.25,-36.5 588.75,-36.5 588.75,-0.5 426.25,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-13.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"506.75,-0.75 506.75,-36.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"547.75\" y=\"-13.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.3466</text>\n",
       "</g>\n",
       "<!-- 1979265014472&#45;&gt;1979265012936 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>1979265014472&#45;&gt;1979265012936</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M575.99,-36.97C593.73,-41.82 613.02,-47.1 631.28,-52.1\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"630.16,-55.42 640.73,-54.68 632.01,-48.67 630.16,-55.42\"/>\n",
       "</g>\n",
       "<!-- 1979236222216 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>1979236222216</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"0,-110.5 0,-146.5 178.25,-146.5 178.25,-110.5 0,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"42.5\" y=\"-123.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"85,-110.75 85,-146.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"131.62\" y=\"-123.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;15.0000</text>\n",
       "</g>\n",
       "<!-- 1979236222216&#45;&gt;1979236221000 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1979236222216&#45;&gt;1979236221000</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M178.71,-139.95C186.64,-140.98 194.7,-142.02 202.7,-143.05\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"202.07,-146.5 212.44,-144.31 202.97,-139.56 202.07,-146.5\"/>\n",
       "</g>\n",
       "<!-- 1979265014216 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>1979265014216</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"629.25,-110.5 629.25,-146.5 791.75,-146.5 791.75,-110.5 629.25,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"669.5\" y=\"-123.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 9.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"709.75,-110.75 709.75,-146.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"750.75\" y=\"-123.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.5000</text>\n",
       "</g>\n",
       "<!-- 1979236228936&#45;&gt;1979265014216 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>1979236228936&#45;&gt;1979265014216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M575.99,-165.03C593.73,-160.18 613.02,-154.9 631.28,-149.9\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"632.01,-153.33 640.73,-147.32 630.16,-146.58 632.01,-153.33\"/>\n",
       "</g>\n",
       "<!-- 1979265012040&#45;&gt;1979265013448 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1979265012040&#45;&gt;1979265013448</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M386.07,-73.5C394.02,-73.5 402.12,-73.5 410.17,-73.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"409.98,-77 419.98,-73.5 409.98,-70 409.98,-77\"/>\n",
       "</g>\n",
       "<!-- 1979265012680 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>1979265012680</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"426.25,-110.5 426.25,-146.5 588.75,-146.5 588.75,-110.5 426.25,-110.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-123.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 2.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"506.75,-110.75 506.75,-146.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"547.75\" y=\"-123.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 3.1192</text>\n",
       "</g>\n",
       "<!-- 1979265012680&#45;&gt;1979265014216 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>1979265012680&#45;&gt;1979265014216</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M589.1,-128.5C598.41,-128.5 607.97,-128.5 617.42,-128.5\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"617.37,-132 627.37,-128.5 617.37,-125 617.37,-132\"/>\n",
       "</g>\n",
       "<!-- 1979265013128 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>1979265013128</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"3.38,-0.5 3.38,-36.5 174.88,-36.5 174.88,-0.5 3.38,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"45.88\" y=\"-13.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;1.0000</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"88.38,-0.75 88.38,-36.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"131.62\" y=\"-13.2\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n",
       "</g>\n",
       "<!-- 1979265013128&#45;&gt;1979265012040 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>1979265013128&#45;&gt;1979265012040</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M160.26,-36.97C178.78,-41.85 198.93,-47.15 217.98,-52.17\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.03,-55.53 227.59,-54.7 218.81,-48.77 217.03,-55.53\"/>\n",
       "</g>\n",
       "<!-- 1979265014216&#45;&gt;1979265015432 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>1979265014216&#45;&gt;1979265015432</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M792.22,-117.6C800.1,-116.53 808.15,-115.44 816.15,-114.36\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"816.48,-117.85 825.92,-113.04 815.54,-110.91 816.48,-117.85\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x1ccd5635508>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1 = Value(5)\n",
    "x2 = Value(2)\n",
    "loss = ((2-x1)**2 + (1 - x2)**2) / 2\n",
    "loss.backward()\n",
    "draw_dot(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多层感知机"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed(2757539057)\n",
    "class neuron:\n",
    "    # 一个神经元\n",
    "    def __init__(self, n_in) -> None:\n",
    "        self.w = [Value(random.uniform(-1, 1)) for _ in range(n_in)]\n",
    "        self.b = Value(random.uniform(-1, 1))\n",
    "\n",
    "    def __call__(self, x):\n",
    "        # 每个神经元的多个输入乘以权重再加上bias\n",
    "        y = sum(wi*xi for wi, xi in zip(self.w, x)) + self.b\n",
    "        out = y.tanh()\n",
    "        return out\n",
    "    \n",
    "    def paramters(self):\n",
    "        return self.w + [self.b]\n",
    "\n",
    "class layer:\n",
    "    # 一层\n",
    "    def __init__(self, n_in, n_out) -> None:\n",
    "        self.neurons = [neuron(n_in) for _ in range(n_out)]\n",
    "\n",
    "    def __call__(self, x):\n",
    "        out = [n(x) for n in self.neurons]\n",
    "        return out[0] if len(out)==1 else out\n",
    "    \n",
    "    def paramters(self):\n",
    "        return [p for neuron in self.neurons for p in neuron.paramters()]\n",
    "\n",
    "class MLP:\n",
    "    def __init__(self, n_in, n_outs) -> None:\n",
    "        alllayer = [n_in] + n_outs\n",
    "        self.layers = [layer(alllayer[i], alllayer[i+1]) for i in range(len(alllayer) - 1)]\n",
    "\n",
    "    def __call__(self, x):\n",
    "        for l in self.layers:\n",
    "            x = l(x)\n",
    "        return x\n",
    "    \n",
    "    def paramters(self):\n",
    "        return [p for l in self.layers for p in l.paramters()]\n",
    "    \n",
    "    def zero_grad(self):\n",
    "        for p in self.paramters():\n",
    "            p.grad = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.0, -1.0, -1.0, 1.0]\n",
      "[Value(0.9828099310029023), Value(0.6739083599577359), Value(0.9888243968938993), Value(0.7989463588220734)]\n"
     ]
    }
   ],
   "source": [
    "xs = [[2.0, 3.0, -1.0],\n",
    "      [3.0, -1.0, 0.5],\n",
    "      [0.5, 1.0, 1.0],\n",
    "      [1.0, 1.0, -1.0]]\n",
    "ys = [1.0, -1.0, -1.0, 1.0]\n",
    "net = MLP(3, [4, 1])\n",
    "predys = [net(x) for x in xs]\n",
    "print(ys)\n",
    "print(predys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value(6.798109744319807)\n",
      "0.19047103083249756\n",
      "-1.1901121702169908\n",
      "1.7833215196622172\n",
      "-1.3785987865733949\n"
     ]
    }
   ],
   "source": [
    "predys = [net(x) for x in xs]\n",
    "loss1 = sum((y - predy)**2 for y, predy in zip(ys, predys))\n",
    "net.zero_grad()\n",
    "loss1.backward()\n",
    "print(loss1)\n",
    "print(net.layers[0].neurons[0].w[2].grad)\n",
    "print(net.layers[0].neurons[3].w[0].grad)\n",
    "print(net.layers[1].neurons[0].w[0].grad)\n",
    "print(net.layers[1].neurons[0].w[2].grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 神经网络在训练时干了什么？\n",
    "\n",
    "我们需要考虑一下这里每一个梯度的意义，也就是说，经过这4个输入我们总结出来  \n",
    "  \n",
    "位于第一层的第一个神经元的第三个权重对应的输入（原始数据）这一批次上来看能够减小损失，因此应该增加其权重  \n",
    "位于第一层的第四个神经元的第一个权重对应的输入（原始数据）这一批次上来看能够增加损失，因此应该减小其权重  \n",
    "位于第二层的第一个神经元的第一个权重对应的输入（可能对应某些特征的数据）这一批次上来看能够增加损失，因此应该减小其权重  \n",
    "位于第二层的第一个神经元的第三个权重对应的输入（可能对应某些特征的数据）这一批次上来看能够减少损失，因此应该减小其权重 \n",
    "\n",
    "由损失函数可以看出损失越小，我们的预测越接近正确答案，这就是神经网络的学习过程 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value(6.798109744319807)\n",
      "Value(0.021896554906918293)\n",
      "Value(0.007903126627045197)\n",
      "Value(0.004709467610690412)\n",
      "Value(0.0033207380352581408)\n",
      "Value(0.0025502475193998194)\n",
      "Value(0.00206262068863621)\n",
      "Value(0.0017272913731853938)\n",
      "Value(0.0014830875312689146)\n",
      "Value(0.001297614291742303)\n"
     ]
    }
   ],
   "source": [
    "# 多次运行此单元格会发现损失降低\n",
    "lr = 0.1\n",
    "for i in range(300):\n",
    "    predys = [net(x) for x in xs]\n",
    "    loss = sum((y - predy)**2 for y, predy in zip(ys, predys))\n",
    "    net.zero_grad()\n",
    "    loss.backward()\n",
    "    # let's do this!\n",
    "    # print(predys)\n",
    "    if (i%30==0):\n",
    "        print(loss)\n",
    "    for p in net.paramters():\n",
    "        p.data += -lr * p.grad # 注意不要直接加p 一是计算图，二是引用不再是net内的数据\n",
    "# 一次训练完成"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Value(0.9886754681986569), Value(-0.9903000144675275), Value(-0.9777731676426495), Value(0.9791246858863208)]\n"
     ]
    }
   ],
   "source": [
    "predys = [net(x) for x in xs]\n",
    "print(predys)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3.7.2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
